{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#解题提示\n",
    "在完成过程中，需要综合运用目前学到的基础知识： \n",
    "深度神经网络 \n",
    "激活函数 \n",
    "正则化 \n",
    "初始化 \n",
    "卷积 \n",
    "池化 \n",
    "并探索如下超参数设置： \n",
    "卷积kernel size \n",
    "卷积kernel 数量 \n",
    "学习率 \n",
    "正则化因子 \n",
    "权重初始化分布参数\n",
    "\n",
    "#批改标准\n",
    "准确度达到98%或者以上60分，作为及格标准，未达到者本作业不及格，不予打分。 \n",
    "使用了正则化因子或文档中给出描述：10分。 \n",
    "手动初始化参数或文档中给出描述：10分，不设置初始化参数的，只使用默认初始化认为学员没考虑到初始化问题，不给分。 \n",
    "学习率调整：10分，需要文档中给出描述。 \n",
    "卷积kernel size和数量调整：10分，需要文档中给出描述。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /root/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-a116e81672e8>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /root/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /root/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /root/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /root/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /root/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = './'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create the model\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y = tf.placeholder(tf.float32, [None, 10]) #真实y值\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-4-02863edf610a>:90: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#学习率非常重要\n",
    "# current_learning_rate = tf.to_float(0.005) #测试集结果：0.9792\n",
    "# current_learning_rate = tf.to_float(0.01)#测试集结果：0.9659\n",
    "# current_learning_rate = tf.to_float(0.001)#测试集结果：0.9873\n",
    "current_learning_rate = tf.to_float(0.0005)#测试集结果：0.9879\n",
    "# current_learning_rate = tf.to_float(0.0001)#测试集结果：0.9767\n",
    "\n",
    "#x reshape 卷积矩阵\n",
    "x_image = tf.reshape(x, [-1, 28,28,1])\n",
    "\n",
    "#卷积\n",
    "shape = [5,5,1,32] #测试集结果：0.9879 \n",
    "#     shape = [10,10,1,32]  #测试集结果：0.9822\n",
    "#     shape = [2,2,1,32]  #测试集结果：0.9819\n",
    "#     shape = [3,3,1,32]  #测试集结果：0.9859\n",
    "#     shape = [6,6,1,32]  #测试集结果：0.9878\n",
    "#     shape = [4,4,1,32]  #测试集结果：0.9868\n",
    "#     shape = [5,5,1,64] #测试集结果：0.9875 \n",
    "    \n",
    "    \n",
    "W_conv = tf.Variable(tf.truncated_normal(shape, stddev=0.1), collections = [tf.GraphKeys.GLOBAL_VARIABLES, 'WEIGHTS'])\n",
    "shape = [32]\n",
    "#     shape = [64]#测试集结果：0.9875 \n",
    "b_conv = tf.Variable(tf.constant(0.1, shape = shape))\n",
    "con = tf.nn.conv2d(x_image, W_conv, strides=[1,1,1,1], padding = 'SAME') + b_conv\n",
    "con_re = tf.nn.relu(con)\n",
    "\n",
    "#池化\n",
    "pool = tf.nn.max_pool(con_re, ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID')\n",
    "\n",
    "\n",
    "#第二次\n",
    "#卷积\n",
    "shape2 = [5,5,32,64] #测试集结果：0.9879 986\n",
    "#     shape2 = [10,10,32,64] #测试集结果：0.9822\n",
    "#     shape2 = [2,2,32,64]  #测试集结果：0.9819\n",
    "#     shape2 = [3,3,32,64]  #测试集结果：0.9859\n",
    "#     shape2 = [6,6,32,64]  #测试集结果：0.9878\n",
    "#     shape2 = [4,4,32,64]  #测试集结果：0.9868\n",
    "#     shape2 = [5,5,64,128] #测试集结果：0.9875\n",
    "\n",
    "W_conv2 = tf.Variable(tf.truncated_normal(shape2, stddev=0.1), collections = [tf.GraphKeys.GLOBAL_VARIABLES, 'WEIGHTS'])\n",
    "shape2 = [64]\n",
    "#     shape2 = [128]\n",
    "b_conv2 = tf.Variable(tf.constant(0.1, shape = shape2))\n",
    "\n",
    "con2 = tf.nn.conv2d(pool, W_conv2, strides=[1,1,1,1], padding = 'SAME') + b_conv2\n",
    "con_re2 = tf.nn.relu(con2)\n",
    "    \n",
    "#池化\n",
    "pool2 = tf.nn.max_pool(con_re2, ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID')\n",
    "\n",
    "#全联接层\n",
    "W_connection = tf.Variable(tf.truncated_normal([7*7*64, 1024], stddev=0.1), \n",
    "#     W_connection = tf.Variable(tf.truncated_normal([7*7*128, 1024], stddev=0.1), \n",
    "                     collections = [tf.GraphKeys.GLOBAL_VARIABLES, 'WEIGHTS'])\n",
    "b_connection = tf.Variable(tf.constant(0.1, shape = [1024]))\n",
    "\n",
    "# 展开\n",
    "flat = tf.reshape(pool2, [-1, 7*7*64])\n",
    "#     flat = tf.reshape(pool2, [-1, 7*7*128])#测试集结果：0.9875 \n",
    "fc = tf.nn.relu(tf.matmul(flat, W_connection) + b_connection)\n",
    "    \n",
    "#drop\n",
    "keep = tf.placeholder(tf.float32)\n",
    "drop = tf.nn.dropout(fc, keep)\n",
    "\n",
    "#第二次全联接层\n",
    "W_conn2 = tf.Variable(tf.truncated_normal([1024,10], stddev=0.1), \n",
    "                      collections = [tf.GraphKeys.GLOBAL_VARIABLES, 'WEIGHTS'])\n",
    "b_conn2 = tf.Variable(tf.constant(0.1, shape = [10]))\n",
    "\n",
    "#     y_ = tf.nn.softmax(tf.matmul(drop, W_conn2) + b_conn2)\n",
    "logits = tf.matmul(drop, W_conn2) + b_conn2\n",
    "\n",
    "#激活 loss reg2\n",
    "# cross_entropy = tf.reduce_mean(\n",
    "#     tf.losses.softmax_cross_entropy(onehot_labels=y, logits=y_))\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits))\n",
    "\n",
    "reg2 = tf.add_n([tf.nn.l2_loss(w) for w in tf.get_collection('WEIGHTS')])\n",
    "total_loss = cross_entropy + 4e-5*reg2\n",
    "\n",
    "\n",
    "#优化器也很重要\n",
    "train_step = tf.train.AdamOptimizer(current_learning_rate).minimize(total_loss)\n",
    "# train_step = tf.train.AdamOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess = tf.Session()\n",
    "sess.run(init_op)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9885\n"
     ]
    }
   ],
   "source": [
    "for current_step in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    loss, l2, total_loss_value, lr,_ = sess.run(\n",
    "           [ cross_entropy, reg2, total_loss, current_learning_rate,train_step], \n",
    "           feed_dict={x: batch_xs, y: batch_ys, keep:0.5})\n",
    "    \n",
    "    \n",
    "# Test trained model\n",
    "correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, y: mnist.test.labels, keep:0.5}))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
